import intake
import ast

from distributed import Client
from ncar_jobqueue import NCARCluster
cluster = NCARCluster()
cluster.scale(10)
client = Client(cluster)

/glade/u/home/mgrover/.local/lib/python3.9/site-packages/dask_jobqueue/core.py:19: FutureWarning: format_bytes is deprecated and will be removed in a future release. Please use dask.utils.format_bytes instead.
  from distributed.utils import format_bytes, parse_bytes, tmpfile
/glade/u/home/mgrover/.local/lib/python3.9/site-packages/dask_jobqueue/core.py:19: FutureWarning: parse_bytes is deprecated and will be removed in a future release. Please use dask.utils.parse_bytes instead.
  from distributed.utils import format_bytes, parse_bytes, tmpfile
/glade/u/home/mgrover/.local/lib/python3.9/site-packages/dask_jobqueue/htcondor.py:6: FutureWarning: parse_bytes is deprecated and will be removed in a future release. Please use dask.utils.parse_bytes instead.
  from distributed.utils import parse_bytes
/glade/work/mgrover/miniconda3/envs/cesm-collections-dev/lib/python3.9/site-packages/distributed/node.py:160: UserWarning: Port 8787 is already in use.
Perhaps you already have a cluster running?
Hosting the HTTP server on port 39863 instead
  warnings.warn(

client

Client

Client-dfbd23e7-0142-11ec-99a0-3cecef1b11fa

Connection method: Cluster object Cluster type: PBSCluster
Dashboard: https://jupyterhub.hpc.ucar.edu/stable/user/mgrover/proxy/39863/status

Cluster Info

 
col = intake.open_esm_datastore(
    "/glade/work/mgrover/cesm-validation-catalog.json",
    csv_kwargs={"converters": {"variables": ast.literal_eval}},
    sep="/",
)
col

None catalog with 8 dataset(s) from 7402 asset(s):

unique
component 1
stream 4
date 2501
case 2
member_id 1
frequency 4
variables 545
path 7402 
cat = col.search(
    variables='TEMP',
    stream='pop.h',
)
cat

None catalog with 2 dataset(s) from 2400 asset(s):

unique
component 1
stream 1
date 1200
case 2
member_id 1
frequency 1
variables 434
path 2400 
sub = cat.search(date=sorted(cat.df.date.unique())[:12])

dsets = sub.to_dataset_dict(cdf_kwargs={'use_cftime': True, 'chunks': {'time': 10}})

--> The keys in the returned dictionary of datasets are constructed as follows:
	'component/stream/case'
100.00% [2/2 00:00<00:00]

Do the actual plottingΒΆ

import xarray as xr
from cartopy import crs
import holoviews as hv

import holoviews as hv
import geoviews as gv
import datashader
from holoviews.operation.datashader import regrid, shade, datashade, rasterize
import matplotlib.pyplot as plt

hv.extension('bokeh')

import panel as pn
pn.extension()

def plot_interactive(ds, variable, cmap='magma'):
    """Plots an interactive plot using holoviews"""
    da = ds[variable]
    da['nlon'] = ds.nlon
    da['nlat'] = ds.nlat
    da = da.drop_vars(['TLAT', 'TLONG', 'ULAT', 'ULONG'])
    
    hvds = hv.Dataset(da, kdims=['time', 'z_t', 'nlon', 'nlat'])
    quadmesh = hvds.to(hv.QuadMesh, kdims=["nlon", "nlat"],dynamic=True)
    datashade_obj = rasterize(quadmesh, precompute=True, cmap=plt.cm.RdBu_r).opts(colorbar=True, width=600, height=400, cmap=cmap, tools=['hover'])

    return datashade_obj.relabel(f'{ds.title}')

def plot_comparison(ds1, ds2, variable):
    diff = ds1[[variable]] - ds2[[variable]]
    diff.attrs['title'] = f'{ds1.title} - {ds2.title}'
    diff[variable].attrs['units'] = ds1[variable].units
    plots = hv.Layout([plot_interactive(ds1, variable) + plot_interactive(ds2, variable) + plot_interactive(diff, variable, cmap='seismic')]).cols(2)
    panel = pn.Column(pn.pane.Markdown(f"## {ds1[variable].name} {ds1[variable].units}", align="center"), plots)
    return panel

ds = dsets['ocn/pop.h/b1850.f19_g17.validation_nuopc.004']
ds2 = dsets['ocn/pop.h/b1850.f19_g17.validation_mct.004']

plot_comparison(ds, ds2, 'TEMP')
<no title>